package com.algo.count;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URL;

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
 * get title from url list
 *
 */
public class GetTitle 
{
	public static String getTitle(String url){
		Document doc;
		
		try {
			new URL(url);
			Connection con = Jsoup.connect(url);
			con.userAgent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)");
			doc = con.get();
		} catch (IOException e) {
			System.out.println(url);
			return null;
		}
		
    	Elements heads = doc.getElementsByTag("head");
    	
    	StringBuilder sb = new StringBuilder();
    	
    	for(Element head : heads){
    		Elements titles = head.getElementsByTag("title");
    		for(Element title : titles){
    			sb.append(title.text());
    		}
    		
    		sb.append("\t");
    		Elements keys = head.getElementsByAttributeValue("name", "keywords");
    		for(Element key : keys){
    			sb.append(key.attr("content"));
    		}
    		
    		sb.append("\n");
    		Elements descs = head.getElementsByAttributeValue("name", "description");
    		for(Element desc : descs){
    			sb.append(desc.attr("content"));
    		}
    	}
    	
        return sb.length() > 3 ?sb.toString().replace("\n", " "):null;//
	}
	public static void getTitles(String in, String out){
		try{
			BufferedWriter bw = new BufferedWriter(new FileWriter(out));
	    	
	    	BufferedReader br = new BufferedReader(new FileReader(in));
	    	String url;
	    	while((url = br.readLine()) != null){
	    		String title = getTitle(url);
	    		if(title != null)
	    			bw.write(url + "\t" + title + "\n");
	    	}
	    	br.close();
	    	bw.close();
		}catch(IOException e){
			throw new RuntimeException("in or out file can not open:" + e);
		}
	}
    public static void main( String[] args )
    {
    	//System.out.print(getTitle("http://www.baike.com/wiki/LVS%E8%B4%9F%E8%BD%BD%E5%9D%87%E8%A1%A1"));
    	getTitles(args[0], args[1]);
    }
}

